# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.exporters import JsonItemExporter
from pymongo import MongoClient
from DoubanBook.items import ItemType
from DoubanBook.common.redis_opt import RedisOpt,RedisConstant
import time

class DoubanbookPipeline(object):

    collection_doub_book = 'doub_book'
    collection_jy_user = 'jy_user_info'

    redisOpt = RedisOpt.get_instance()

    @classmethod
    def from_crawler(cls,crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DB')
        )

    def __init__(self,mongo_uri,mongo_db):
        self.fp = open("json.json", 'wb')  # wb方式打开json文件
        self.exporter = JsonItemExporter(self.fp, ensure_ascii=False, encoding='utf-8')
        self.exporter.start_exporting()

        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    def open_spider(self, spider):
        spider.logger.info("爬虫开始了...")
        self.client = MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):
        if len(item):
            if ItemType.DOUB_BOOK.value == item['itemName']:
                insert_data = {
                    'rate': item['rate'],
                    'title': item['title']
                }
                book_tb = self.db[self.collection_doub_book]
                book_tb.insert_one(insert_data)
            elif ItemType.JIAY_USER.value == item['itemName']:
                #self.redisOpt.l_rpush(name=RedisConstant.JIAYUAN_USER_LIST_KEY)
                spider.logger.info('process_item  item>>>{}'.format(item))
            elif ItemType.JIAY_USER_LIST_ITEM.value == item['itemName']:
                insert_data = {
                    'uid':item['uid'],
                    'realUid':item['realUid'],
                    'nickName':item['nickName'],
                    'sex':item['sex'],
                    'age':item['age'],
                    'marriage':item['marriage'],
                    'height':item['height'],
                    'education':item['education'],
                    'image':item['image'],
                    'workLocation':item['workLocation']
                }

                jy_user_tb = self.db[self.collection_jy_user]
                spider.logger.info('jy_user_tb.count()>>>{}'.format(jy_user_tb.count()))
                if jy_user_tb.find_one({'uid':item['uid']}) is None:
                    spider.logger.info('添加新纪录  item>>>{}'.format(insert_data))
                    jy_user_tb.insert_one(insert_data)
                else:
                    spider.logger.info('uid>>>{}已经存在跳过'.format(item['uid']))

            elif ItemType.PY89IP.value == item['itemName']:
                print('--')
                iplist = item['proxy89ipList']
                spider.logger.info('iplist>>>{}'.format(iplist))
                ip_port_list = []
                for x in iplist:
                    x = x.replace('\n','')
                    if 2 == len(x.split(':')):
                        ip_port_list.append(RedisConstant.proxy89ip_list_item_format(x.split(':')[0], x.split(':')[1]))

                if len(ip_port_list) > 0:
                    for item in ip_port_list:
                        self.redisOpt.l_rpush(RedisConstant.PROXY_IP_89_LIST_KEY, item)

                    self.redisOpt.key_expire(name=RedisConstant.PROXY_IP_89_LIST_KEY,expire=3000)
                    self.redisOpt.s_set(key = RedisConstant.JIAYUAN_PORXY_STATUS,value='1',expire= 3600)
                else:
                    spider.logger.info('%s >>>> 没有遍历出格式正确的ip和端口' % spider.name)
        return item

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.fp.close()
        self.client.close()
        spider.logger.info("爬虫结束了!>>>{}".format(spider.name))
        if 'proxy89ip' == spider.name :
            self.redisOpt.s_set(RedisConstant.PROXY_IP_89_RUN_STATUS,'0',1800)
        elif 'jiayuan' == spider.name:
            self.redisOpt.key_del(RedisConstant.PROXY_IP_89_LIST_KEY)